import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;


public class MorningStar {
	public static void main(String[] args){
		 FileInputStream stream;
			try {
				stream = new FileInputStream("infile.txt");
				DataInputStream in = new DataInputStream(stream);
				BufferedReader br = new BufferedReader(new InputStreamReader(in));
				String strLine;
				FileWriter fstreamall = new FileWriter("cleaned/CleanAll.csv");
				BufferedWriter outall = new BufferedWriter(fstreamall);
				FileInputStream rstream = new FileInputStream("ratios.txt");
				DataInputStream rin = new DataInputStream(rstream);
				BufferedReader rbr = new BufferedReader(new InputStreamReader(rin));
				String toWrite="";
				ArrayList<String> Ratios=new ArrayList<String>();
				outall.write("Companies,");
				while ((strLine = rbr.readLine()) != null)   
				{	
					
					Ratios.add(strLine);
					toWrite+=strLine+",";
					//outall.write(strLine.replaceAll("%", "")+",");
				}
				outall.write(toWrite.substring(0, toWrite.lastIndexOf(",")).replaceAll("%", ""));
				outall.write("\n");
				while ((strLine = br.readLine()) != null)   
				{	
					init(strLine,outall,Ratios);
					
				}
				rbr.close();
				rstream.close();
				stream.close();
				br.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			
	}
    public static ArrayList<String> betas = new ArrayList<String>();
    
	public static void init(String cName,BufferedWriter outall,ArrayList<String> Ratios) throws IOException{
		File f = new File(cName+".csv");
		String ticker=getTicker(cName);

     	
		boolean exists=true;
		if(!f.exists()) { 

			exists=getCSV(ticker,cName);
			

		}
		if(exists){
			FileInputStream rstream;
			try {
			
				try {
					
					betas.add(getBeta(ticker));
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
				
			//	outall.write("Companies,");
				//for(int r=0;r<Ratios.size();r++)
					//outall.write(Ratios.get(r)+",");
				//outall.write("\n");
				//outall.flush();
				printRatios(cName+".csv", Ratios,betas.get(betas.size()-1),outall);
			//	rbr.close();
		//		rstream.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		
	}
	public static boolean getCSV(String ticker,String cName)
	  {
	     try
	     {
	        /*
	         * Get a connection to the URL and start up
	         * a buffered reader.
	         */
	        long startTime = System.currentTimeMillis();
	 
	        System.out.println("Connecting to Morning Star...\n");
	        if(cName=="")
	        	return false;
	        URL url = new URL("http://financials.morningstar.com/ajax/exportKR2CSV.html?t="+ticker);
	        url.openConnection();
	        InputStream reader = url.openStream();
	 
	        /*
	         * Setup a buffered file writer to write
	         * out what we read from the website.
	         */
	        FileOutputStream writer = new FileOutputStream(cName+".csv");
	        byte[] buffer = new byte[153600];
	        int totalBytesRead = 0;
	        int bytesRead = 0;
	 
	        System.out.println("Reading CSV file 150KB blocks at a time.\n");
	 
	        while ((bytesRead = reader.read(buffer)) > 0)
	        {  
	           writer.write(buffer, 0, bytesRead);
	           buffer = new byte[153600];
	           totalBytesRead += bytesRead;
	        }
	 
	        long endTime = System.currentTimeMillis();

	        if(totalBytesRead<=0){
	        	System.out.println("Company Not Found\n");
	        	File f = new File(cName+".csv");
	    		if(f.exists()) { 
	    			reader.close();
	    			writer.close();
	    			f.delete();
	    			return false;
	    		}
	        }
	        else{
	        System.out.println("Done. "+cName+".csv " + (new Integer(totalBytesRead).toString()) + " bytes read (" + (new Long(endTime - startTime).toString()) + " millseconds).\n");
	        
	        reader.close();
	       
			
	        writer.close();
	        return true;
	        }
	        
	     }
	     catch (MalformedURLException e)
	     {
	        e.printStackTrace();
	     }
	     catch (IOException e)
	     {
	        e.printStackTrace();
	     }
	     return false;
	 
	  }
    public static void writeCSV(String[] value,BufferedWriter out,BufferedWriter all)
    {
    	
    	try {
		
		
		for (int i=0;i<value.length-1;i++)
		{
			out.write(value[i] + ",");
		}
		if(value.length>=11){
		try{
		Double.parseDouble(value[10]);
		System.out.println("WRITING:"+value[value.length-1]+",");
		
		all.write(value[10]);
		}catch(NumberFormatException ex){
			all.write("0");

		}
		}
		out.write("\n");
		all.flush();
		out.flush();
		
	} catch (IOException e) {

		e.printStackTrace();
	}
    }
    public static void writeyearsCSV(String[] value,BufferedWriter out)
    {
    	
    	try {
		
		
		for (int i=0;i<value.length-1;i++)
		{
			out.write(value[i] + ",");
		}
		out.write("\n");
		out.flush();
		
	} catch (IOException e) {

		e.printStackTrace();
	}
    }
	public static String[] getRatio(String name,String strLine){
		String[] years=null;
		if (strLine.contains(name))
		{	
			years = strLine.split(","); // Separated by commas
			System.out.println(years[0]);
			System.out.print("Years: ");

			for(int i=1;i<years.length;i++)
				System.out.print(years[i]+"\t");
			System.out.println("");
		}
		return years;

	}
	 public static void printRatios(String fileName,ArrayList<String> Ratios,String Beta,BufferedWriter all) throws IOException{
         FileInputStream fstream;

         	FileWriter fstream1 = new FileWriter("cleaned/Clean"+fileName);
			BufferedWriter out = new BufferedWriter(fstream1);
			int lineNO=0;
         try 
         {
                 fstream = new FileInputStream(fileName);
                 DataInputStream in = new DataInputStream(fstream);
                 BufferedReader br = new BufferedReader(new InputStreamReader(in));
                 String strLine;
                 boolean[] isRatio=new boolean[Ratios.size()];
                 String[] toWrite;
                 System.out.println("Calculating for "+fileName);
                 all.write(fileName.replaceAll(".csv", "")+",");
                 while ((strLine = br.readLine()) != null)   
                 {      	if(++lineNO==3)	writeyearsCSV(strLine.split(","), out);

                         for(int i=0;i<Ratios.size();i++){
                                 if(!isRatio[i]&&(toWrite=getRatio(Ratios.get(i), strLine))!=null){
                                         isRatio[i]=true;
                                         
                                         writeCSV(toWrite, out,all);
                                         if(i!=Ratios.size()-1)
                                        	 all.write(",");
                                         
                                 }
                         }
                         //System.out.println(strLine);
                         
                 }
                 all.write("\n");
                 all.flush();
                 out.write("Beta,"+Beta);
                 out.flush();
                 out.close();
         } 
         catch (Exception e) 
         {
                 // TODO Auto-generated catch block
                 e.printStackTrace();
         }
 }
	public static String getTicker(String companyName){
		String ticker="";
		//http://d.yimg.com/autoc.finance.yahoo.com/autoc?query=apple&callback=YAHOO.Finance.SymbolSuggest.ssCallback
		   URL tickerUrl;
		try {
			tickerUrl = new URL("http://d.yimg.com/autoc.finance.yahoo.com/autoc?query=" + companyName + "&callback=YAHOO.Finance.SymbolSuggest.ssCallback");
			URLConnection tickerConn;
			try {
				tickerConn = tickerUrl.openConnection();
				BufferedReader tickerIn = new BufferedReader(new InputStreamReader(tickerConn.getInputStream()));
		        String tempr,tempw="";
		        while((tempr=tickerIn.readLine())!=null){
		        	tempw+=tempr;
		        }
		        String what="\"symbol\":\"";
		        //System.out.println(tempw.indexOf(what));
		        int i=tempw.indexOf(what);
		        if(i>=0)
		        	for(i+=what.length();i<tempw.length();i++)
		        	{
		        		if(tempw.charAt(i)=='"')
		        			break;
		        		ticker+=tempw.charAt(i);
		        	}
			
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
	          

		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
           
				return ticker;
	}
	public static String getBeta(String ticker) throws IOException{

		    URL mb =  new URL("http://in.finance.yahoo.com/q/ks?s=" +ticker );
            URLConnection ucon = mb.openConnection();
            BufferedReader in2 = new BufferedReader(new InputStreamReader(ucon.getInputStream()));
            String s1;

          boolean  done = false;


            while ((s1=in2.readLine()) != null && done == false)
            {
                //Console.WriteLine(s);

//===========================================================================================================================================
            	
            	
            	if (s1.contains("Beta:") /*&& yes == true*/)
                {

            		int index=s1.indexOf("Beta:");
            	//	s1=s1.substring(index+5,s1.length());
            		//System.out.println(s1);
                    String pattern = "(?i)(<td.*?>)(.+?)(</td>)";
                    String updated = s1.replaceAll(pattern, "$2"+";"); 
                    pattern="((<td>.*?</td>)|<!\\[CDATA\\[.*?\\]\\]>|<!--.*?-->|<.*?>)";

                    
                    updated=updated.replaceAll(pattern, "$2");
                    String[] elems=updated.split(";");
                    for(int i=0;i<elems.length;i++)
                    {
                    	if(elems[i].equals("Beta:")){
                    		updated=elems[i+1];
                    		break;
                    	}
                    }
                    System.out.println("Beta:"+updated);
                    return updated;
                }
            	
            	
            }
            return "";
	}
}
